eval 'exec perl -wS $0 ${1+"$@"}'
 if 0;

use strict;
use warnings;
use POSIX qw(strftime);

use DBI();
use File::Basename;

#use File;


# Following variable is used as a general file descriptor for all the 
# output of the current tool (the log file). Actually, this file will
# contain the copy of both the stdout and stderr channel.
my $G_LOG;

my $G_ARCHIVE_PATH="/data/samba_share/archives/tct-huizhou";
#my $G_ARCHIVE_PATH="/data/test";
my $G_OPL_PATH="/data/samba_share/tct-huizhou/opl_ns/imported";

################################################################################
#
################################################################################
sub get_time
  {
    my $separator=shift @_;
    my $now_time="";

    if (! defined $separator)
      {
	$separator=":";
      }

    $now_time = strftime("%H".$separator."%M".$separator."%S",localtime());

    return $now_time;
  }

################################################################################
#
################################################################################
sub get_date
  {
    my $separator=shift @_;
    my $now_date="";

    if (! defined $separator)
      {
	$separator="/";
      }

    $now_date = strftime("%Y".$separator."%m".$separator."%d",localtime());

    return $now_date;
  }

################################################################################
#
################################################################################
sub mylog
{
    if ($G_LOG)
    {
	print $G_LOG get_date()." ".get_time()." ".join("",@_);
    }
}

################################################################################
#
################################################################################
sub error
  {
    print STDERR get_date()." ".get_time()." [ERROR] ",@_;
    mylog "[ERROR] ", @_;
  }

################################################################################
#
################################################################################
sub warning
  {
    print STDERR get_date()." ".get_time()." [WARNING] ",@_;
    mylog("[WARNING],",@_);
  }

################################################################################
#
################################################################################
sub debug
  {
    my $h_ref_param=shift @_;

    if (exists $h_ref_param->{DEBUG})
      {
	print STDOUT get_date()." ".get_time()." [DEBUG] ",@_;
	mylog("[DEBUG] ",@_);
      }
  }

################################################################################
#
################################################################################
sub info
{
    print STDOUT get_date()." ".get_time()." [INFO] ",@_;
    mylog("[INFO] ",@_);
}

################################################################################
#
################################################################################
sub print_h_content
{
    my $h_ref_param=shift @_;

    debug $h_ref_param, "Printing Global Hash Content:\n";
    debug $h_ref_param, "######################################################\n";
    foreach my $thekey (keys %$h_ref_param)
    {
	debug $h_ref_param, $thekey." => ".$h_ref_param->{$thekey}."\n";
    }
    debug $h_ref_param, "######################################################\n";
}

################################################################################
#
################################################################################
sub db_connect
{
    my $h_ref_param=shift @_;

    my $dbh = DBI->connect("DBI:mysql:database=".$h_ref_param->{DBNAME}.";host=localhost",
                         $h_ref_param->{DBLOGIN}, $h_ref_param->{DBPASSWD},
                         {'RaiseError' => 0});

    $h_ref_param->{DB}=$dbh;

    return (defined $dbh);
}

################################################################################
#
################################################################################
sub db_disconnect
{
    my $h_ref_param=shift @_;
    $h_ref_param->{DB}->disconnect();
}


################################################################################
#
################################################################################
sub db_archive_already_processed

{
    my $h_ref_param=shift @_;
    my $archive = shift @_;
    my $prodday = shift @_;

    my $sth = $h_ref_param->{DB}->prepare("SELECT COUNT(*) FROM imported_archive where archive_name='$archive' and prodday='$prodday';");
    $sth->execute();
    my $ref = $sth->fetchrow_hashref();

    $sth->finish();

    return $ref->{'COUNT(*)'}!=0;
}

################################################################################
#
################################################################################
sub db_archive_insert
{
    my $h_ref_param=shift @_;
    my $archive = shift @_;
    my $prodday = shift @_;
    my $insert_date = shift @_;
    my $status = shift @_;
    my $lc_name = shift @_;
    my $md5sum = shift @_;

    my $res=0;

    debug($h_ref_param,"archive=>$archive<\n");
    debug($h_ref_param,"prodday=>$prodday<\n");
    debug($h_ref_param,"insert date=>$insert_date<\n");
    debug($h_ref_param,"status=>$status<\n");
    debug($h_ref_param,"lc_name=>$lc_name<\n");
    debug($h_ref_param,"md5sum=>$md5sum<\n");

    my $sth = $h_ref_param->{DB}->prepare("INSERT INTO imported_archive (archive_name,prodday,datint,status,lc, md5) VALUES ( '$archive', '$prodday', '$insert_date', '$status', '$lc_name', '$md5sum');");
#    my $sth = $h_ref_param->{DB}->prepare("INSERT INTO imported_archive (archive_name,prodday,datint,status,lc) VALUES ( '$archive', '$prodday', '$insert_date', '$status', '$lc_name');");

    $res=$sth->execute();

    return $res;
}

################################################################################
#
################################################################################
sub get_remote_file_list
{
    my $h_ref_param=shift @_;

    my $remote_host=shift @_;
    my $remote_login=shift @_;
    my $remote_passwd=shift @_;
    my $remote_dir=shift @_;
    my $cmd="";

    my @output=();
    my @result=();

    info "Listing archives in sn_Produce_data/$remote_dir...\n";

    $cmd="/data/scripts/ftp_getlist.sh 2>&1 $remote_host $remote_login '$remote_passwd' 'sn_Produce_data/$remote_dir'";
        
    debug $h_ref_param, "Executing in 'get_remote_file_list' function:>$cmd<\n";
    
    @output=`$cmd`;
    
    if ($? != 0)
    {
	error("Failed to execute >$cmd<. Previous error produced the following output:\n");
	error(@output);
	return @result;
    }

    foreach my $line (@output)
    {
	chomp($line);
	print $line. "\n";
	
	if ($line=~/No such file or directory/)
	{
	    warning "Directory >$remote_dir< was not found on remote host.\n";
	    @result=();
	    return @result;
	}

	if ($line=~/.+Connection refused/)
	{
	    error "Failed to connect to the tct-huizhou FTP Server.\n";
	    @result=();
	    return @result;
	}

	if ($line=~/^\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(.+)\.rar$/)
	{
	    push(@result,$1.".rar");
	}
    }
    
    return @result;
}

################################################################################
#
################################################################################
sub get_remote_file
{
    my $h_ref_param=shift @_;
    my $remote_host=shift @_;
    my $remote_login=shift @_;
    my $remote_passwd=shift @_;
    my $remote_path=shift @_;
    my $remote_file=shift @_;
    my $local_dir=shift @_;

    my $cmd="";
    my @output=();

    $cmd="cd $local_dir;/data/scripts/ftp_getfile.sh 2>&1 $remote_host $remote_login '$remote_passwd' 'sn_Produce_data/$remote_path' '$remote_file'";
   
    debug $h_ref_param, "Executing in 'get_remote_file' function:>$cmd<\n";
    @output=`$cmd`;

    if ( ($? != 0) || (! -f "$local_dir/$remote_file") )
    {
	error("Failed to execute >$cmd<. Produced output is:\n");
	error(@output);
	error("End of output. Skipping current file processing...\n");
	return 0;
    }
    
    # Success !
    return 1;
}

################################################################################
#
################################################################################
sub file_unrar
{
    my $h_ref_param=shift @_;
    my $dest_dir = shift @_;
    my $file = shift @_;
    my $cmd ="";
    my @output=();

    #$cmd="cd $dest_dir; /data/tools/bin/unrar x -p$h_ref_param->{ARCHIVE_PASSWD} \"$file\"";
    $cmd="cd $dest_dir; /data/tools/sources/unrar/unrar x -p$h_ref_param->{ARCHIVE_PASSWD} \"$file\"";
    debug $h_ref_param, "Executing in 'test_file_integrity' function:>$cmd<\n";
    @output=`$cmd`;
    
    if ($? != 0)
    {
	error("Archive $file seems to be corrupted. Produced test output is:\n");
	error(@output);
	return 0;
    }
    
    return 1;
}

################################################################################
#
################################################################################
sub file_get_md5sum
{
    my $h_ref_param=shift @_;
    my $file=shift @_;
    my $cmd="";
    my $res="";

    $cmd="md5sum \"$file\" | awk '{ print \$1}'";
    debug $h_ref_param, "Executing in 'file_get_md5sum' function:>$cmd<\n";

    $res=join("",`$cmd`);
    chomp($res);

    if ($res!~/^[0-9a-f]{32}$/)
    {
	error("file_get_md5sum seem to be not working on >$file<.\n");
	$res="00000000000000000000000000000000";
    }

    return $res;
}

################################################################################
#
################################################################################
sub file_move
{
    my $h_ref_param=shift @_;
    my $source_file=shift @_;
    my $dest_file=shift @_;
    my $file_type=shift @_;

    my $cmd="";
    my @output=();

    $cmd="mv -f \"$source_file\" \"$dest_file\"";
    debug $h_ref_param, "Executing in 'file_move' function:>$cmd<\n";
    @output=`$cmd`;

    if ($? != 0)
    {
	error("Failed to move $file_type to its final location. Produced operation output is:\n");
	error(@output);
	return 0;
    }

    return 1;
}

################################################################################
#
################################################################################
sub file_chmod
{
    my $h_ref_param=shift @_;
    my $mode=shift @_;
    my $file=shift @_;

    my $cmd="";
    my @output=();

    $cmd="chmod $mode \"$file\"";
    debug $h_ref_param, "Executing in 'file_chmod' command:>$cmd<\n";
    @output=`$cmd`;

    if ($? != 0)
    {
	error("Failed to chmod $file. Produced operation output is:\n");
	error(@output);
	return 0;
    }

    return 1;
}


################################################################################
#
################################################################################
sub get_data_file
{
    my $h_ref_param=shift @_;
    my $extract_file_path=shift @_;
    my $file_type1=shift @_;
    my $file_type2=shift @_;

    my @output=();

    my $cmd="";

    $cmd="find $extract_file_path \\( -name \\*.$file_type1 -o -name \\*.$file_type2 \\) -print";
    debug $h_ref_param, "Executing in 'get_data_file' function:>$cmd<\n";
    @output=`$cmd`;

    if ( ($? != 0) )
    {
	error("Failed to execute >$cmd<. Produced output is:\n");
	error(@output);
	error("End of output. Skipping current file processing...\n");
	return 0;
    }
    
    # Success !
    return @output;
}

################################################################################
#
################################################################################
sub process
{
    my $h_ref_param=shift @_;
    my $err=0;
    my $total_processed_archive_counter=0;
    my $skipped_processed_archive_counter=0;
    my $error_processed_archive_counter=0;
    my $new_processed_archive_counter=0;
    my $md5="";

    print_h_content($h_ref_param);

    if (! db_connect($h_ref_param))
    {
	error("Failed to connect to database.\n");
	return 1;
    }

    # Checking lock file existence first in order to avoid to run two s    # instance of the tool in parallel.
    info "Checking lock file /tmp/retrieve_data_tct-huizhou_ns.lock... \n";
    if ( -f "/tmp/retrieve_data_tct-huizhou_ns.lock")
    {
	info "Checking lock exits! Aborting... \n";
	return 1;
    }

    info "Creating lock file... \n";
    `touch "/tmp/retrieve_data_tct-huizhou_ns.lock"`;
    if ($? != 0)
    {
	error("Failed to create lock file >$h_ref_param->{LOCK_FILE}< \n");
	return 1;
    }

    foreach my $mydir (@{$h_ref_param->{PROD_DAY_LIST}})
    {
	my @filelist=();

	@filelist=get_remote_file_list($h_ref_param, $h_ref_param->{RHOST},$h_ref_param->{RLOGIN},$h_ref_param->{RPASSWD},$mydir);

	if (scalar @filelist == 0)
	{
	    info "$mydir not existing or no file found in $mydir...\n";
	    next;
	}

	foreach my $archive (@filelist)
	{
	    my $status="";

	    $total_processed_archive_counter++;

	    $status="failed";

	    chomp($archive);
	    $archive=~s/sftp> //;

	    if (db_archive_already_processed($h_ref_param,$archive,$mydir))
	    {
		info("Archive $archive already retrieved. Skipping...\n");
		$skipped_processed_archive_counter++;
		next;
	    }
	    info "Start to download $archive from sn_Produce_data/$mydir...\n";

	    if (!get_remote_file($h_ref_param, $h_ref_param->{RHOST},$h_ref_param->{RLOGIN},$h_ref_param->{RPASSWD},"$mydir", "$archive",$h_ref_param->{WORK_DIR}))
	    {
		# Error message already printed into the previous function.
		$err++;
		next;
	    }

	    info "File Download Terminated.\n";

	    info "Start to extract $archive... \n";

	    if (! file_unrar($h_ref_param,"$h_ref_param->{WORK_DIR}","$h_ref_param->{WORK_DIR}/$archive"))
	    {
		error("Current file will be skipped... \n");
		$error_processed_archive_counter++;
		$err++;
		next;
	    }

	    info "File Extract Terminated.\n";
	    
	    my ($name,$path,$suffix) = fileparse("$h_ref_param->{WORK_DIR}/$archive",(".rar"));
	    my @oplfilelist=();

	    info "Try to find OPL file...\n";

	    @oplfilelist = get_data_file($h_ref_param,"$h_ref_param->{WORK_DIR}/$name","000","000");
	    	    
	    if (scalar @oplfilelist == 0)
	    {
		error("OPL File Not Found in $archive. \n");
		$status="failed";
	    } else {

		info "OPL File Found. \n";
		
		foreach my $oplfile (@oplfilelist)
		{
		    chomp($oplfile);
		    my ($oplname,$oplpath,$oplsuffix) = fileparse("$oplfile",());
		    info "Start to move OPL file $oplname...\n";

		    if (! file_move($h_ref_param, "$oplfile", "$G_OPL_PATH","OPL file '$oplname'"))
		    {
			error("OPL file $oplname couldn't be copied to final location.\n");
		
		    } 
		    else 
		    {
			file_chmod($h_ref_param, "o+r,o+w", "$G_OPL_PATH/$oplname");
			info "OPL File Move Terminated. \n";
			$status="imported";
		    }
		} # foreach my oplfile (@oplfilelist)
	    }

	    $md5=file_get_md5sum($h_ref_param,"$h_ref_param->{WORK_DIR}/$archive");

	    if (! db_archive_insert($h_ref_param,$archive,$mydir, $h_ref_param->{START_TIME}, $status,"tct-huizhou", $md5))
	    {
		error("Too bad! Couldn't register current archive as being already $status.\n");
		error("Next time maybe...\n");
		$error_processed_archive_counter++;
		$err++;
		next;
	    }

	    if (! file_move($h_ref_param,"$h_ref_param->{WORK_DIR}/$archive", "$G_ARCHIVE_PATH/$status/$mydir"."_"."$archive","archive"))
	    {
		error("Archive couldn't be copied to final location.\n");
		$error_processed_archive_counter++;
		$err++;
		next;
	    }

	    $new_processed_archive_counter++;
	    

	} # foreach my $archive (@filelist)
	
    } # foreach my $mydir ()

    info "Delete lock file... \n";
    `rm -f "/tmp/retrieve_data_tct-huizhou_ns.lock"`;

    info("### Summary: #################################################\n");
    info("Total Archive Processed: $total_processed_archive_counter\n");
    info("       Included skipped: $skipped_processed_archive_counter\n");
    info("         Included error: $error_processed_archive_counter\n");
    info("           Included new: $new_processed_archive_counter\n");
    info("##################################\n");

    if (! db_disconnect($h_ref_param))
    {
	error("Failed to disconnect from database.\n");
	return 1;
    }

    return $err;
}

################################################################################
#
################################################################################
sub print_version
{
  my $h_ref_param=shift @_;
  print $h_ref_param->{EXECUTABLE_NAME}." ".$h_ref_param->{VERSION}." ".$h_ref_param->{RELEASE_DATE}."\n";
}

################################################################################
#
################################################################################
sub parse_args
{
    my $h_ref_param=shift @_;

    my $arg="";
    my $err=0;

    while ($arg=shift @ARGV)
      {
	  chomp($arg);
        if ( $arg =~ /^-/ )
          {
            if ( ($arg =~ /^-h/) || ($arg =~ /^--help/))
              {
                print_usage($h_ref_param);
                exit(0);
              }
            elsif ( ($arg =~ /^-d/) || ($arg =~ /^--debug/))
              {
                $h_ref_param->{DEBUG}=1;
              }
            elsif ( ($arg =~ /^-v/) || ($arg =~ /^--version/))
              {
                print_version();
		exit(0);
              }
            elsif ( ($arg =~ /^-l/) || ($arg =~ /^--log/))
              {
		  $arg=shift @ARGV;
		  $h_ref_param->{LOG}=$arg;
              }
	    else
	      {
                error "Unexpected option >$arg<.\n";
                $err++;
                next;
	      }
          }
	  else
	  {
	      #<rhost>: Mandatory. Remote host address IP from which to retrieve the data.\n";
	      if (! exists $h_ref_param->{RHOST})
	      {
		  $h_ref_param->{RHOST}=$arg;
	      }
	      #<rlogin>: Mandatory. Remote host login to use to connect with FTP.\n";
	      elsif (! exists $h_ref_param->{RLOGIN})
	      {
		  $h_ref_param->{RLOGIN}=$arg;
	      }
	      #<rpasswd>: Mandatory. Remote host password to use to connect with FTP.\n";
	      elsif (! exists $h_ref_param->{RPASSWD})
	      {
		  $h_ref_param->{RPASSWD}=$arg;
	      }
	      #<dbname>: Mandatory. Local database name to use to check for already downloaded archives.\n";
	      elsif (! exists $h_ref_param->{DBNAME})
	      {
		  $h_ref_param->{DBNAME}=$arg;
	      }
	      #<dblogin>: Mandatory. Local database login to use <dbname> database.\n";
	      elsif (! exists $h_ref_param->{DBLOGIN})
	      {
		  $h_ref_param->{DBLOGIN}=$arg;
	      }
	      #<dbpasswd>: Mandatory. Local database password to use <dbname> database.\n";
	      elsif (! exists $h_ref_param->{DBPASSWD})
	      {
		  $h_ref_param->{DBPASSWD}=$arg;
	      }
	      #<archivepasswd>: Mandatory. Archive password.
	      elsif (! exists $h_ref_param->{ARCHIVE_PASSWD})
	      {
		  $h_ref_param->{ARCHIVE_PASSWD}=$arg;
	      }
	      #<production day> [<production day> ...]: Mandatory. At least one production day to retrieve.\n";
	      else 
	      {
		  if (! exists $h_ref_param->{PROD_DAY_LIST})
		  {
		      my @locallist=();
		      $h_ref_param->{PROD_DAY_LIST}=\@locallist;
		  }

		  push( @{$h_ref_param->{PROD_DAY_LIST}}, $arg);
	      }
	  }
      }
    
    if (! exists $h_ref_param->{PROD_DAY_LIST})
    {
        error "Missing mandatory arguments. Use -h or --help for usage.\n";
	$err++;
    }

    if ($err>0)
      {
        return 1;
      }

    return 0;
}

################################################################################
#
################################################################################
sub print_usage
  {
    my $h_ref_param=shift @_;

    print "USAGE: ".$h_ref_param->{EXECUTABLE_NAME}." [-h|--help] [-d|--debug] [-v|--version] [-l|--log <log file>] ...] <rhost> <rlogin> <rpasswd> <dbname> <dblogin> <dbpasswd> <archive passwd> <production day> [<production day> ...]\n";
    print "\n";
    print "PURPOSE:\n";
    print "   Download the production data of the given day\n";
    print "\n";
    print "PARAMETERS DESCRIPTION:\n";
    print "   <rhost>: Mandatory. Remote host address IP from which to retrieve the data.\n";
    print "\n";
    print "   <rlogin>: Mandatory. Remote host login to use to connect with FTP.\n";
    print "\n";
    print "   <rpasswd>: Mandatory. Remote host password to use to connect with FTP.\n";
    print "\n";
    print "   <dbname>: Mandatory. Local database name to use to check for already downloaded archives.\n";
    print "\n";
    print "   <dblogin>: Mandatory. Local database login to use <dbname> database.\n";
    print "\n";
    print "   <dbpasswd>: Mandatory. Local database password to use <dbname> database.\n";
    print "\n";
    print "   <archivepasswd>: Mandatory. Archive password.\n";
    print "\n";
    print "   <production day> [<production day> ...]: Mandatory. At least one production day to retrieve.\n";
    print "               <production day> format should be yymmdd. Ex: 070903 for 3rd of Sept. 2007\n";
    print "\n";
    print "OPTIONS DESCRIPTION:\n";
    print "   [-h|--help]:\n";
    print "      Print this usage message and exit.\n";
    print "\n";
    print "   [-v|--version]:\n";
    print "      Print the version number and exit.\n";
    print "\n";
    print "   [-d|--debug]:\n";
    print "      Print excessive output to understand the processing.\n";
    print "\n";
    print "\n";
  }
################################################################################
#
################################################################################
sub open_log
{
    my $h_ref_param=shift @_;

    if (! open($G_LOG,">".$h_ref_param->{LOG}))
    {
	warning "Cannot create log file ".$h_ref_param->{LOG}." in current directory.\n";

	$h_ref_param->{LOG}="/tmp/".$h_ref_param->{EXECUTABLE_NAME}.".log";

	if (! open($G_LOG,">".$h_ref_param->{LOG}))
	{
	    warning "Still cannot create alternative log file ".$h_ref_param->{LOG}.". No log will be performed.\n";
	    $G_LOG=0;
	}
    }
}

################################################################################
#
################################################################################
sub intro
  {
    my $h_ref_param=shift @_;
    my $err=0;
    my $date_for_file="";

    # Updating some global variables.

    $h_ref_param->{START_TIME}=get_date("-")." ".get_time(":");

    $h_ref_param->{DATE_OF_DAY}=substr(get_date(""),2);

    $date_for_file=get_date("-")."_".get_time("-");

    $h_ref_param->{WORK_DIR}="/tmp/retrieve_data_tct-huizhou_ns.".$date_for_file;

    `mkdir -p $h_ref_param->{WORK_DIR}`;
    if ($? != 0)
    {
	error("Failed to create temporary directory ".$h_ref_param->{WORK_DIR}."\n");
	$err++;
    }

    # The following is the default log file name. Can be overriden 
    # with the -l or --log option.

    $h_ref_param->{VERSION}="0.1.0";
    $h_ref_param->{EXECUTABLE_NAME}="retrieve_data_tct-huizhou_ns.pl";

    $h_ref_param->{LOG}=$h_ref_param->{EXECUTABLE_NAME}.".$date_for_file.log";

    #$h_ref_param->{LOCK_FILE}="/tmp/".$h_ref_param->{EXECUTABLE_NAME}.".lock";
    
    $h_ref_param->{RELEASE_DATE}="2007/10/31";

    # Following instruction forces flushing for each output to STDOUT
    $|=1;

    return $err;
  }

################################################################################
#
################################################################################
sub outtro
{
    my $h_ref_param=shift @_;
    my $err=0;

    `rm -Rf $h_ref_param->{WORK_DIR}`;
    if ($? != 0)
    {
	error("Failed to remove temporary directory ".$h_ref_param->{WORK_DIR}."\n");
	$err++;
    }

    if ($G_LOG)
    {
	close($G_LOG);
    }

    return $err;
}

################################################################################
#
################################################################################
sub main
{
  # This hash list will be used to store input parameters and some other
  # usefull information.
  # The actual valable keys are:
  #   VERSION:           string: current version of the tool.
  #   START_TIME:        string: start time of the script in the form YY_MM_DD.HH.MM.SS
  #   EXECUTABLE_NAME:   string: name of the current script.
  #   LOG:               string: name of the log file.
  #   RELEASE_DATE:      string: date of current tool release in form: YYYY/MM/DD
  #   DEBUG:             boolean: if this key exists, then the current script is run in
  #                               debug mode.
  #
  my %params=();
  
  if (intro(\%params)!=0)
  {
      error "Error while executing intro... Aborting.\n";
      exit(1);
  }

  if (parse_args(\%params)!=0)
  {
      error "Error while parsing arguments... Aborting.\n";
      exit(1);
  }

  open_log(\%params);

  if (process(\%params)!=0)
  {
      error "Error while processing... Aborting.\n";
  }

  if (outtro(\%params)!=0)
  {
      error "Error while executing outtro... Aborting.\n";
      exit(1);
  }

}


################################################################################
#
################################################################################

main();

exit ;

